{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Reinforcement Learning in Action\n",
    "### by Alex Zai and Brandon Brown\n",
    "\n",
    "#### Chapter 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Supplemental"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def moving_average(x,step=5,window=50):\n",
    "    num = (x.shape[0] - window) / step\n",
    "    num = int(num)\n",
    "    avg = np.zeros(num)\n",
    "    slider = np.ones(window) / window\n",
    "    start = 0\n",
    "    for i in range(num):\n",
    "        end = start + window\n",
    "        avg[i] = slider @ x[start:end]\n",
    "        start = start + step\n",
    "    return avg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gym import envs\n",
    "#envs.registry.all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "env = gym.make('CartPole-v0')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "state1 = env.reset()\n",
    "action = env.action_space.sample()\n",
    "state, reward, done, info = env.step(action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    " \n",
    "l1 = 4\n",
    "l2 = 150\n",
    "l3 = 2\n",
    " \n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.Softmax(dim=0)\n",
    ")\n",
    " \n",
    "learning_rate = 0.009\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = model(torch.from_numpy(state1).float())\n",
    "action = np.random.choice(np.array([0,1]), p=pred.data.numpy())\n",
    "state2, reward, done, info = env.step(action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discount_rewards(rewards, gamma=0.99):\n",
    "    lenr = len(rewards)\n",
    "    disc_return = torch.pow(gamma,torch.arange(lenr).float()) * rewards\n",
    "    disc_return /= disc_return.max()\n",
    "    return disc_return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(preds, r):\n",
    "    return -1 * torch.sum(r * torch.log(preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 4.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_DUR = 200\n",
    "MAX_EPISODES = 500\n",
    "gamma = 0.99\n",
    "score = []\n",
    "for episode in range(MAX_EPISODES):\n",
    "        curr_state = env.reset()\n",
    "        done = False\n",
    "        transitions = []\n",
    "\n",
    "        for t in range(MAX_DUR):\n",
    "            act_prob = model(torch.from_numpy(curr_state).float())\n",
    "            action = np.random.choice(np.array([0,1]), p=act_prob.data.numpy())\n",
    "            prev_state = curr_state\n",
    "            curr_state, _, done, info = env.step(action)\n",
    "            transitions.append((prev_state, action, t+1))\n",
    "            if done:\n",
    "                break\n",
    " \n",
    "        ep_len = len(transitions)\n",
    "        score.append(ep_len)\n",
    "        reward_batch = torch.Tensor([r for (s,a,r) in\n",
    "        transitions]).flip(dims=(0,))\n",
    "        disc_rewards = discount_rewards(reward_batch)\n",
    "        state_batch = torch.Tensor([s for (s,a,r) in transitions])\n",
    "        action_batch = torch.Tensor([a for (s,a,r) in transitions])\n",
    "        pred_batch = model(state_batch)\n",
    "        prob_batch = pred_batch.gather(dim=1,index=action_batch.long().view(-1,1)).squeeze()\n",
    "        loss = loss_fn(prob_batch, disc_rewards)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x121725fd0>]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1f3/8dcnOwESSAgQCBiWALIJEjZFtC4VV7Raq21dsXRxqdpWbf22tv3W1vp1qa3+bKni0lqUqlXcRWsF2QPIvoWEJYHsELKRbc7vjxltIgkJ2SaZeT8fDx6ZOffOzYd5TN65Offcc8w5h4iIBJYQfxcgIiJtT+EuIhKAFO4iIgFI4S4iEoAU7iIiASjM3wUA9OnTxyUnJ/u7DBGRLmXt2rUFzrmEhrZ1inBPTk4mLS3N32WIiHQpZra3sW3qlhERCUAKdxGRAKRwFxEJQAp3EZEApHAXEQlATYa7mQ0ys4/NbKuZbTGzH/ra48xssZnt8n3t7Ws3M/ujmaWb2UYzO7W9/xMiIlJfc87ca4AfOedGA9OAW8xsNHAv8JFzLgX4yPcc4AIgxfdvLvBUm1ctIiLH1eQ4d+fcQeCg73GJmW0DBgKzgbN8uz0P/Ae4x9f+gvPOJbzSzHqZWaLvOCIiAaGyppajVZ4vnlfVeqioqqWsqoayyhoKy6oo8v2rrK5t9DipyXHMHNHgfUitckI3MZlZMjARWAX0qxPYOUA/3+OBwP46L8vytdULdzObi/fMnsGDB59g2SIi/rN27yFueHY1JUdrmrW/WePbvnfmMP+Gu5n1AF4F7nDOHbE61TrnnJmd0Kofzrl5wDyA1NRUrRgiIl3CjpwSbnpuDXHdI7jj3BF8noRhoUZ0RBjdI0KJjgwjLjqC+B4RxHWPICo8tMPrbFa4m1k43mB/0Tn3mq859/PuFjNLBPJ87dnAoDovT/K1iYh0afsKy7n2mVVEhYfw9zlTGRQX7e+SGtVkuJv3FP0ZYJtz7tE6mxYB1wMP+r6+Uaf9VjN7CZgKFKu/XUS6gsqaWj7cmkdFA33kzjme+DidyhoPC787vVMHOzTvzP104Fpgk5l95mv7Gd5QX2hmc4C9wFW+be8AFwLpQDlwY5tWLCLSDjwexx0vfca7m3Ma3adHZBgvzJnCyP49O7CylmnOaJlPgcYuB5zTwP4OuKWVdYmIdKjfvrONdzfncM+sUVw8PrHBfXpFh9MzKryDK2uZTjHlr4iIPz23LJOnP83khtOS+d6ZQ7HjDW/pIjT9gIgEtQ+25PCrt7Zy3uh+/Pzi0QER7KBwF5EgtmJ3IbctWM/4gbE8fvUEQkMCI9hB4S4iQWrD/sPc/PwaBsdF8+yNU4iOCKxeaoW7iASdnbklXP/sauJ6RPC3OVOJ6x7h75LanMJdRIJKcXk13356FRGhIbw4Zxr9Y6P8XVK7CKy/Q0REmvDeloPklVTyyvemMzi+c9+I1Bo6cxeRoPLOphwGxXVj0km9/V1Ku1K4i0jQKK6oZvnuAi4YmxgwQx4bo3AXkaDx0bZcqmsds8b293cp7U7hLiJB493NOSTGRjEhqZe/S2l3CncRCQqllTV8sjOf88f0JySAblZqjMJdRILCx9vzqKrxcEEQdMmAwl1EgsR7m3Po0yOC1OQ4f5fSIRTuIhLwKqpq+XhHHueP6R9Q88ccj8JdRALeJzvzKa+q5YKxDc/THogU7iIS8N7ceIDe0eFMHRocXTLQjHA3s/lmlmdmm+u0TTCzlWb2mZmlmdkUX7uZ2R/NLN3MNprZqe1ZvIhIU4rKqvhgSw6XTRxIeGjwnM8253/6HDDrS20PAb9yzk0AfuF7DnABkOL7Nxd4qm3KFBFpmdfWZVFd67h68mB/l9Khmgx359wSoOjLzUCM73EscMD3eDbwgvNaCfQys+Dp5BKRTsU5x4LV+5g4uFeXWNS6LbV0Vsg7gPfN7GG8vyBO87UPBPbX2S/L13bwywcws7l4z+4ZPDi4fqOKSMdYu/cQu/PLeOiK8f4upcO1tAPq+8CdzrlBwJ3AMyd6AOfcPOdcqnMuNSEhoYVliIg0bsHq/XSPCOWi8cHXgdDScL8eeM33+J/AFN/jbGBQnf2SfG0iIh3qyNFq3t50gEsnDKR7ZPAtXdHScD8AnOl7fDawy/d4EXCdb9TMNKDYOXdMl4yISHt747MDHK32cM2UQU3vHICa/HVmZguAs4A+ZpYF3A98B3jczMKAo/j6zoF3gAuBdKAcuLEdahYROS7nHC+t3sfJiTGMGxjr73L8oslwd85d08imSQ3s64BbWluUiEhrLN1VwJYDR3jg8rEBvyhHY4JnRL+IBAXnHI8s3snAXt24clKSv8vxG4W7iASUj7blsWH/YW47eziRYaH+LsdvFO4ickIKSiu56s8r+GBLjr9LOYbH43h08U5Oio/miiA+a4eW38QkIkGo1uO4fcF6Vu8pYm9RGTNS+hAd0Xli5P0tOWw9eIRHrzolqOaRaUhw/+9F5IQ8ungHy3cXct30k8g9Uslfl2T6u6Qv1PrO2ocldGf2hIH+LsfvFO4i0iwfbs3lyY93c82UQfx69lguGNufvyzZTd6Ro/4uDYC3Nh5gV14pd5w7ImgW5DgehbuINCk9r4S7Fn7G2IEx3H/JGADuvWAU1bUeHvlgp5+r8/a1P/lxOil9e3DRuOCbaqAhCncRaVRNrYe/fLKbi/74KaEhxlPfmkRUuHcEyknx3bluejIL1+5n28Ejfq1z8bZcduaWcstXhhOis3ZA4S4ijdiec4SvPbWc3727nZkjEnjvjpkMiouut8/tZ6cQ2y2ce1/bRK6fumec8561D46L5uIgnCCsMQp3ETlG9uEKrvrzCrIPVfDENycy79pJ9IuJOma/2Ohw/nf2WLYfPMK5j37CwrT9eG9U7zhLdhWwMauY7581jLAgHyFTl94JEamn1uO46+XPqPU4XvvBaVw8fsBxb+G/5JQBvHfHTE7uH8Pdr2zk+mfXsDKjEI+nY0L+yX+nkxgbxddO1QiZuhTuIlLP00szWJVZxP2XjuGk+O7Nes2QPt15ae40fj17DOv2HuLqeSs546GP+b/3t5NT3H7dNaszi1i9p4i5M4cG9d2oDVG4i8gXNmcX8/AHO7hgbH++foJ3eIaEGNdNT2b1fefwh29MYHjfHjz1n91cP391u5zFezyOxxbvJL57RNCtj9ocCncRAeBodS13vPwZcd0j+O3l41o8m2J0RBiXTRzI8zdN4Q9XT2RHbglvbWr7ZR3+vGQ3KzIKufO8EXSL0Fn7lyncRQSARxfvJD2vlIe/fgq9u0e0yTEvHpfIyH49+cOHO6mp9bTJMQFWZhTy8Ps7uGh8It+aqrP2hijcRYQN+w/z9NIMrpkymDNS2m5N45AQ487zUsjIL+ONzw60yTHzSyq5fcF6kuO78/srxgftfO1N6Twz/kin55zjkQ92criiir49o+jbM5JJJ/UmpV9Pf5cmrVBV4+HuVzbSt2cUP71wVJsf//wx/RkzIIbHP9rFpRMG1JvQq9bj2FdUzq7cEnLqjJP3eByllTUUV1RzuLwaM7yfuZhI3tp4kOKKal6YM4UeQbg2anM1Z5m9+cDFQJ5zbmyd9tvwrrpUC7ztnLvb1/5TYI6v/Xbn3PvtUbh0vEUbDvDEx+n0iAyjtLIGgIiwEN6+bYYCvgt78uN0duSWMP+GVGKiwtv8+GbGXeeNYM7zaby6NotzR/fjrQ0HWLThAJsPHKGqpvHumm7hofSKDsfjHAWlVdT6Lsw+dOV4RvWPafNaA4k1dcOBmc0ESoEXPg93M/sKcB9wkXOu0sz6OufyzGw0sACYAgwAPgRGOOdqj/c9UlNTXVpaWuv/N9JuyqtqOPvhT+jTM4I3bplBda2HfUXlXD1vJQN7deO1H5wW9FOsdiTnHD95ZSORYSHcf8kYIsIaf+8zC8p447NsiiuqKTlaQ3lVDYmx3RiW0IOYbmHc+fJnXDgukcevntiu9V7+/5azK7eEyhoPNR7H6MQYZqT0YXjfHozo15OBvbpRd+aAHlFh9YY31nochWWVeDzQP/bYG6qCkZmtdc6lNrStOWuoLjGz5C81fx940DlX6dsnz9c+G3jJ155pZul4g35FC2uXTuKp/+wm58hRnvjmREJDjNCQUEb068kDl43l+y+u48mP07nj3BH+LjNovLXxIK+szQJgT2EZT317UoNn3XsLy/j6n1dQUFpJz8gwYrqFExUewr+353G02nvGHN894ovJwNqLmXHfRSfz89c3c+bIBL42MYmR/U/sr73QEKNvT4V6c7W0w2oEcIaZPQAcBX7snFsDDARW1tkvy9d2DDObC8wFGDxYV7s7s/1F5fxlSQaXnjKA1OS4etsuGJfI5RMH8qd/p3P2qL6MT+rlpyqDR2llDb95eytjB8Zw3fRkfvbaJq768wqeu3FKvTPavJKjXPvMamo9Hj6860yG9+3xxTaPx3GguIL0vFKSekcT10ajY45ncnIc790xs92/j3i19O/oMCAOmAb8BFhoJ3jJ2jk3zzmX6pxLTUhou6vz0vZ++842Qs0avdj2y0vHkNAjkrsWbqCi6rg9cNIGHv9wJ3kllfzv7LFclTqIZ2+cTNahCi7+06c8+O52NmUVc+RoNTfMX0NBaSXP3jilXrCDdxRLUu9ozhrZ95htEhhaGu5ZwGvOazXgAfoA2cCgOvsl+dqki1qVUci7m3P4wVnDSIzt1uA+sd3CeejK8aTnlfKNeSvIPlzRwVUGjx05JcxftoerJw9i4uDeAJyRksDC705n9IAYnl6awSVPfMrUBz5iZ24Jf/72JCYM0l9Twail4f468BUAMxsBRAAFwCLgajOLNLMhQAqwui0KFf9YmJZFz6gwvjNz6HH3mzkigXnXTiIzv4xL/vQpy9ILOqjC4FBWWUNmQRk/f30zMVFh3H1+/b+iRg+I4YWbprDmvnN56IrxnJHShz9dM5GZI/RXcbBqzlDIBcBZQB8zywLuB+YD881sM1AFXO+8w262mNlCYCtQA9zS1EgZ6byqajws3prDeaP7fbFAw/F8dUx/3ri1B9/921qufWYVv7x0DNdNT27/QgPUvsJyHnxvG0t2Fnwx9BTg91eMa/QO0t7dI7hq8iCumjyowe0SPJozWuaaRjZ9u5H9HwAeaE1R0jmsyCjkyNEaLhzb/AUQhib04PVbTueHL63nl4u2MKp/DFOGxDX9QvlCaWUNT/w7nfmfZhIWalw+cSBJvaPp2zOS5D7dmXRSb3+XKF2Abu+SRr276SDdI0KZkdLnhF7XPTKMP1w9kYv/uJQ7XlrPOz88g17R7T8aIxDkHjnK7CeWkXPkKFecmsTds0Y2uEiGSFN014k0qKbWwwdbcznn5OZ1yXxZj8gw/nTNqeSXVnLvq5s6fHWersg5xz2vbuRwRRWvfn86j1x1ioJdWkzhLg1anVlEUVkVF47r3+JjjEuK5Sfnj+S9LTn8Y/W+NqwuML20Zj//2ZHPvbNGMekkdWVJ66hbRhr0zuaDdAsP5cwRfVt1nJtnDGXprgJ+/eZWJgzqxZgBsfW2O+fIOlRBfmklhaVVlFXW8JVRfYnt1vZznHRm+4vK+c1bWzltWLwuQkubULjLMWo9jvc25/KVUQmtXgQhJMR49KoJXPKnT5n7wloW3Xo68T0iAW/Xz09e2ci/1te/FWLMgBj+8Z1pQRPwHo/jx//cgJnx0JXjCQnRFLbSeuqWkWOs3XuIgtJKLjiBUTLHk9AzknnXTaKgtJLvv7iO6loPNbUe7ly4gX+tz+a7M4fy7A2TeeOW03nym6eyM7eEG59dTVmd4X+B6sjRau55dSOrMov4xSWjSeod7e+SJEDozF2O8c6mg0SEhfCVUa3rkqlrfFIvHrxiHHe+vIH7F22huLyatzcd5KcXjOK7Zw77Yr9TBvUiNARu+cd6bn4+jWdvnNyiC7qdnXOO97fk8Is3tlBQWsn3zhx2wmuWihyPwl2O8eG2XGam9GnzhRAun5jEtoMlzFuSAcD/XHQyN59x7J2vs8Ym8vDXa7lr4QZu/cd65l07KaC6Kjwexx0vf8aiDQcYnRjD09enasI1aXMKd6nnwOEKsg5VcNPpQ9rl+PfMGkVldS2jEmO4Zkrjs4FePjGJw+XV/OrNrfxlSQbfP2tYo/t2NU99sptFGw5w+9nDue2cFM2DL+1C4S71rNlTBNBud5WGhhi/mj226R2BG05LZu3eQzz8wQ5Sk3szObnrDw9cu/cQjy7eyUXjE7nzvBFa/1PajU4ZpJ5VmUX0jAzj5ET/L2FmZvzua+MYHBfNbf9YT2Fppb9LapXiimpuX7CexNgofve1cQp2aVcKd6lndWYRqcm9Ce0kfdw9o8J54psTKSqv4s6FG/B4Gr7TtbyqhlUZhfz5k93cvmA9b2440OSxq2s9HXbnrHOOn762kZwjR/njNRPbZa1SkbrULSNfKCytJD2vlK+d2uDiWX4zZkAs918ymvv+tZkbn1vDo1ed8sVY+aPVtTzywQ6eXbaHGl/wx3YLZ9GGA+zIKeGu80bUuxhbXFHNf3bk8cHWXD7ZkU+/mEiev2lKi4cg/m3lXlZmFHL/xaPp28hUAc45Hl28k3c25XDPrFGcOlgTf0n7U7jLF9bsOQTA1E44i+M3pwzGOfj1W1u58I9L+dM1pxJicPcrG8koKOPrk5KYNbY/Ewb1omdUOP/z+iae+DidzMIyHrhsLJ+mF/D6+gN8sjOP6lpHnx6RXDC2P+9tyeEbf1nJP74zlZPiu59QTR9uzeUXb2zGOVi5u5BHrjqFs0bWHz7q8Th+/dZWnlu+h2+kDuK7TcyLL9JWrDNM6JSamurS0tL8XUbQ+/WbW3lx1V42/fJ8IsI6Z4/dlgPF3PqP9ewtLMMBA2K78dCV4zl9eP2ZK51zzFuSwYPvbfc9h34xkVwyfgAXjEtk4qBehIQYm7OLufaZVUSEhfDizdOaveTcztwSLn9yGUMTevC7r43jx//cwPacEm6eMYRLJwxgUO9oekaFcferG3ltXTY3zxjCfRedrH52aVNmttY5l9rgNoW7fO7iPy2lZ2Q4C+ZO83cpx1VaWcMDb28lKjyUH3115HHH43+8PY+luwo49+S+TB0a3+C1hB05JXzr6ZVU1XgY3rcHYaEhRISGcOWkJC6beGwX1aGyKmY/uYyK6lrevHUG/WOjOFpdy2/e3srfV/53grTIsBAqazz86LwR3Hr2cAW7tDmFuzSp5Gg1p/zqA249O4W7zhvh73I63O78Uh79YCdHjlZTVeMhr6SSPYVlPPnNU7lw3H+nYSitrGHOc2tYv/8wL8+d9sU6pnWPk55Xyv6icrIOVTBxcC9mT+hc1zAkcBwv3JuzzN584GIgzzk39kvbfgQ8DCQ45wrMe2ryOHAhUA7c4Jxb19r/gLS/tXsP4XGds7+9IwxL6MGT3zr1i+cVVbV86+mV3PHyZ8R3j2Dq0Hj2FJTxnRfSyCgo49GrTjkm2D8/zrCE5nXtiLSn5nSsPgfM+nKjmQ0CvgrUnaj7AryLYqcAc4GnWl+idITVmUWEhRgTB+s2eIBuEaE8c/1kBvXuxs0vpPG3FXu49IlPKSit5IWbpuhsXDq9JsPdObcEKGpg02PA3UDdfp3ZwAvOayXQy8zaZmpBaVdr9hQxLimW6AgNoPpc7+4RPH/TFLqFh/LzN7YwoFc3Ft0645iLtyKdUYt+ks1sNpDtnNvwpYtEA4H9dZ5n+doONnCMuXjP7hk8uPE5RqT9Ha2uZcP+Ym48PdnfpXQ6Sb2jefHmqby7OYebzxiiX37SZZzwJ9XMooGf4e2SaTHn3DxgHngvqLbmWNI6S3cVUFXrabf5ZLq6lH49SenX099liJyQlpyGDAOGAJ+ftScB68xsCpANDKqzb5KvTTopj8fxyAc7OCk+mpkjEvxdjoi0kRO+U8U5t8k519c5l+ycS8bb9XKqcy4HWARcZ17TgGLn3DFdMtJ5vLEhm+05Jfz4qyM19axIAGnyp9nMFgArgJFmlmVmc46z+ztABpAO/BX4QZtUKe2isqaWh9/fydiBMVw0Tte9RQJJk90yzrlrmtieXOexA25pfVnSEf6+ch/Zhyv4/RValFkk0Ojv8CB15Gg1T/x7F2ek9GFGiob2iQQahXuQ+uuSDA6VV3PPrFH+LkVE2oHCPQgdLq/i2WV7uGhcImMHxvq7HBFpBwr3IDR/2R5KK2u47Zzh/i5FRNqJwj3IFFdU8+yyTGaN6c+o/v5fJ1VE2ofCPcg8v3wPJUd11i4S6BTuQaTkaDXPfJrJuSf3Y8wA9bWLBDKFexB5YcVeiiuquV1n7SIBT+EeJI5W1/L00gzOGpnA+CTN2S4S6BTuQSJtzyEOlVdz3fST/F2KiHQAhXuQWL67gNAQY8qQeH+XIiIdQOEeJFZkFHJKUiw9IrXYhEgwULgHgdLKGjZmFTN9mM7aRYKFwj0IrMksotbjOG2YJggTCRYK9yCwfHcBEaEhTDqpt79LEZEOonAPAisyCpk4uBdR4aH+LkVEOkhzVmKab2Z5Zra5Ttv/mdl2M9toZv8ys151tv3UzNLNbIeZnd9ehUvzHC6vYsuBI+qSEQkyzTlzfw6Y9aW2xcBY59x4YCfwUwAzGw1cDYzxveb/mZlOF/1oVWYRzqGLqSJBpslwd84tAYq+1PaBc67G93QlkOR7PBt4yTlX6ZzLxLuW6pQ2rFdO0IrdhUSFhzBhkO5KFQkmbdHnfhPwru/xQGB/nW1Zvjbxk+W7C5icHEdEmC6viASTVv3Em9l9QA3wYgteO9fM0swsLT8/vzVlSCPySyrZmVuqLhmRINTicDezG4CLgW8555yvORsYVGe3JF/bMZxz85xzqc651ISEhJaWIcexMqMQgOlDFe4iwaZF4W5ms4C7gUudc+V1Ni0CrjazSDMbAqQAq1tfprTEioxCekSGMU7rpIoEnSYnGjGzBcBZQB8zywLuxzs6JhJYbGYAK51z33PObTGzhcBWvN01tzjnatureDm+FbsLmTokjrBQ9beLBJsmw905d00Dzc8cZ/8HgAdaU5S03oHDFWQWlPHtaZriVyQY6ZQuQC3f7e1vP00XU0WCksI9QC3fXUBc9whG9uvp71JExA8U7gHIOceK3YVMHxpPSIj5uxwR8QOFewDaU1jOweKjGt8uEsQU7gFoWXoBoP52kWCmcA9AK3YXkhgbxZA+3f1dioj4icI9wHg8jhUZhUwfFo/vHgQRCUIK9wCzI7eEorIqzd8uEuQU7gHm8/HtupgqEtwU7gFmxe4CkuOjGdirm79LERE/UrgHkFqPY1VGkc7aRUThHki2HTxCSWUN0zTFr0jQU7gHkNWZ3tUQJyfH+bkSEfE3hXsAWbOniIG9ujFA/e0iQU/hHiCcc6zOLGLqEJ21i4jCPWBkFJRRWFbFZIW7iKBwDxhrfP3tUxTuIkIzwt3M5ptZnpltrtMWZ2aLzWyX72tvX7uZ2R/NLN3MNprZqe1ZvPzX6swi+vSIYKjmkxERmnfm/hww60tt9wIfOedSgI98zwEuwLsodgowF3iqbcqUpqzeU8Tk5DjNJyMiQDPC3Tm3BCj6UvNs4Hnf4+eBy+q0v+C8VgK9zCyxrYqVhh04XEHWoQoNgRSRL7S0z72fc+6g73EO0M/3eCCwv85+Wb62Y5jZXDNLM7O0/Pz8FpYh4B0CCepvF5H/avUFVeecA1wLXjfPOZfqnEtNSEhobRlBbXVmET0iwzg5McbfpYhIJ9HScM/9vLvF9zXP154NDKqzX5KvTdrR6swiJp3Um1CtlyoiPi0N90XA9b7H1wNv1Gm/zjdqZhpQXKf7RtrBobIqduWVqktGROoJa2oHM1sAnAX0MbMs4H7gQWChmc0B9gJX+XZ/B7gQSAfKgRvboWapY7X620WkAU2Gu3PumkY2ndPAvg64pbVFSfMtSy8gOiKU8Umx/i5FRDoR3aHaxX26q4CpQ+KIDAv1dyki0oko3LuwrEPlZBSUMSNFo41EpD6Fexf26a4CAGamaDFsEalP4d6FLd1VQL+YSIb37eHvUkSkk1G4d1G1Hsey3QWckZKg+WRE5BgK9y5qc3Yxh8urOUNdMiLSAIV7F/Vpure//fThCncROZbCvYtauiuf0Ykx9OkR6e9SRKQTUrh3QWWVNazde4gzRuisXUQapnDvglZnFlFd6zhjuMa3i0jDFO5d0JJd+USGhZCa3NvfpYhIJ6Vw74KWpRcwZUgcUeGackBEGqZw72LySyrZmVuqUTIiclwK9y5m+W7vEMjThsX7uRIR6cwU7l3Mit2FxESFMWaApvgVkcYp3LuY5bsLmTY0XkvqichxKdy7kP1F5ewrKleXjIg0qVXhbmZ3mtkWM9tsZgvMLMrMhpjZKjNLN7OXzSyirYoNdit2FwJwmi6mikgTWhzuZjYQuB1Idc6NBUKBq4HfA48554YDh4A5bVGoeC+m9ukRSYqm+BWRJrS2WyYM6GZmYUA0cBA4G3jFt/154LJWfg8BnHMs313IacPiNcWviDSpxeHunMsGHgb24Q31YmAtcNg5V+PbLQsY2NDrzWyumaWZWVp+fn5Lywgau/NLySupVH+7iDRLa7plegOzgSHAAKA7MKu5r3fOzXPOpTrnUhMSNEdKU5Z/3t8+TP3tItK01nTLnAtkOufynXPVwGvA6UAvXzcNQBKQ3coaBVieXkhS724Mjo/2dyki0gW0Jtz3AdPMLNq8ncDnAFuBj4ErfftcD7zRuhKl1uNYkVGoLhkRabbW9LmvwnvhdB2wyXesecA9wF1mlg7EA8+0QZ1BbUPWYYorqtUlIyLNFtb0Lo1zzt0P3P+l5gxgSmuOK/X9a102kWEhnH1yX3+XIiJdhO5Q7eSOVteyaMMBZo3tT0xUuL/LEZEuQuHeyX20LY/iimqunJTk71JEpAtRuHdyr67LIjE2Sv3tInJCFO6dWN6Ro3yyM5/LJw7ULJAickIU7p3Y659lU+txXKEuGRE5QQr3Tso5xytrszh1cC+GJWiiMBE5MQr3Tmpz9hF25pZy5aRB/i5FRLoghXsntO3gEf7njc1EhAukZZsAAAlxSURBVIVw0fhEf5cjIl1Qq25ikvpqaj28vekgf12aQXhoCL+5bOxx1zp1zlFcUU1oiBERFkJFVS2PLd7J31buJaZbOP935Xhiu2lsu4icOIV7G8g9cpQPtuby9NIM9haWM7xvD3KPVDL7iWXcdnYKP/jKMMJD6/+RlHWonJ/9azNLdtaf7jjE4FtTT+JHXx1Br2gtYiUiLaNwb6H0vBL+vnIfn6YXkJ5XCsApSbH87NpJnHdyP4orqvnlm1t47MOdvLv5IJdOGMDUIfGMGRDDy2v28/v3tmPA7eekEBMVRmWNh5pax7mj+x73bF9EpDnMOefvGkhNTXVpaWl+raGqxsOiDQd4bnkm1TWO6cPimT4snmlD4omNrt818uraLO57fRMAU4fEc/rweE4f3ofRiTHHrJL03uYcHvlgB7t8vwBCQ4xaj2PmiAR+e/lYknprCl8RaRkzW+ucS21wW7CHe3Wth/mfZjJ/WSa5RyoZ1b8nCT0jSdtziIrqWkJDjDNS+nDZhIGcOSKBh97fzoLV+5k2NI4/XjORvj2jmvV9CkorSdtTxLp9hxkzIIZLTxmg5fJEpFUU7o1wzvGjf27gtXXZzBjeh+/MHMrMlD6YGVU1HjZkHeajbXm8ueEA2YcrvnjdLV8Zxp3njiAsVIONRMR/jhfuQd3n/uB723ltXTZ3nTeC289JqbctIiyEyclxTE6O4+7zR5K29xAfbc/ltGF9OHOElgUUkc4taMP96aUZ/OWTDK6ddhK3nT38uPuGhBhThsQxZUhcB1UnItI6Qdmv8Pr6bH7z9jYuHNefX146Rn3fIhJwWhXuZtbLzF4xs+1mts3MpptZnJktNrNdvq+926rYtvDxjjx+/M8NTB8az2PfmKDZFkUkILX2zP1x4D3n3CjgFGAbcC/wkXMuBfjI97xTWLfvED/4+zpG9u/JvOsmERkW6u+SRETaRYvD3cxigZn4FsB2zlU55w4Ds4Hnfbs9D1zW2iLbwq7cEm56bg39YiJ57sYp9NSSdSISwFpz5j4EyAeeNbP1Zva0mXUH+jnnDvr2yQH6NfRiM5trZmlmlpafn9/QLm2muLya6+avJjw0hL/NmUpCz8h2/X4iIv7WmnAPA04FnnLOTQTK+FIXjPMOom9wIL1zbp5zLtU5l5qQ0L5DCx98bzt5JZXMv34yg+J0R6iIBL7WhHsWkOWcW+V7/gresM81s0QA39e81pXYOmv3FrFg9T5uPC2ZcUmas0VEgkOLw905lwPsN7ORvqZzgK3AIuB6X9v1wButqrAVqms93PevzSTGRnHneSP8VYaISIdr7U1MtwEvmlkEkAHciPcXxkIzmwPsBa5q5fdoseeW7WF7Tgl/uXYS3SOD9n4tEQlCrUo859xnQEPzGpzTmuO2hezDFTy6eCfnntyXr45u8JquiEjACtg7VB9+fwcOpztQRSQoBWS478ot4fXPsrnhtCGaL11EglJAhvtjH+6ke0QY35051N+liIj4RcCF++bsYt7ZlMNNM4bQu7vWIBWR4BRw4f7Y4p3Edgtnzowh/i5FRMRvAirc1+07xEfb85g7cyix3TR3jIgEr4AJ91qP4/fvbie+ewQ3nJbs73JERPwqIMLdOcev39zCqswifnL+SN2wJCJBLyDC/a9LM3h+xV5unjGEq6cM9nc5IiJ+1+XDfdGGA/z2ne1cND6Rn114sr/LERHpFLp0uK/MKOTHCzcwJTmOR75+CiFaMk9EBOji4d47OoKpQ+OYd90kosK1ZJ6IyOe69JXHkf178rc5U/1dhohIp9Olz9xFRKRhCncRkQCkcBcRCUAKdxGRANTqcDezUDNbb2Zv+Z4PMbNVZpZuZi/7luATEZEO1BZn7j8EttV5/nvgMefccOAQMKcNvoeIiJyAVoW7mSUBFwFP+54bcDbwim+X54HLWvM9RETkxLX2zP0PwN2Ax/c8HjjsnKvxPc8CBjb0QjOba2ZpZpaWn5/fyjJERKSuFt/EZGYXA3nOubVmdtaJvt45Nw+Y5ztWvpntbWEpfYCCFr42UOk9qU/vx7H0ntTXVd+Pkxrb0Jo7VE8HLjWzC4EoIAZ4HOhlZmG+s/ckILupAznnElpahJmlOedSW/r6QKT3pD69H8fSe1JfIL4fLe6Wcc791DmX5JxLBq4G/u2c+xbwMXClb7frgTdaXaWIiJyQ9hjnfg9wl5ml4+2Df6YdvoeIiBxHm0wc5pz7D/Af3+MMYEpbHLeZ5nXg9+oq9J7Up/fjWHpP6gu498Occ/6uQURE2pimHxARCUAKdxGRANSlw93MZpnZDt88Nvf6u56OZmaDzOxjM9tqZlvM7Ie+9jgzW2xmu3xfe/u71o6k+Y7qM7NeZvaKmW03s21mNj2YPyNmdqfv52WzmS0ws6hA/Ix02XA3s1DgSeACYDRwjZmN9m9VHa4G+JFzbjQwDbjF9x7cC3zknEsBPvI9Dyaa76i+x4H3nHOjgFPwvjdB+Rkxs4HA7UCqc24sEIp3KHfAfUa6bLjjHZGT7pzLcM5VAS8Bs/1cU4dyzh10zq3zPS7B+0M7EO/78Lxvt6Ca30fzHdVnZrHATHxDkp1zVc65wwTxZwTvKMFuZhYGRAMHCcDPSFcO94HA/jrPG53HJhiYWTIwEVgF9HPOHfRtygH6+aksf2jxfEcBagiQDzzr66p62sy6E6SfEedcNvAwsA9vqBcDawnAz0hXDnfxMbMewKvAHc65I3W3Oe9Y16AY71p3viN/19KJhAGnAk855yYCZXypCybIPiO98f7VMgQYAHQHZvm1qHbSlcM9GxhU53mz5rEJNGYWjjfYX3TOveZrzjWzRN/2RCDPX/V1sM/nO9qDt5vubOrMd+TbJ9g+J1lAlnNule/5K3jDPlg/I+cCmc65fOdcNfAa3s9NwH1GunK4rwFSfFe5I/BeFFnk55o6lK8/+Rlgm3Pu0TqbFuGd1weCaH4fzXd0LOdcDrDfzEb6ms4BthKknxG83THTzCza9/Pz+fsRcJ+RLn2Hqm9Gyj/gveI93zn3gJ9L6lBmNgNYCmziv33MP8Pb774QGAzsBa5yzhX5pUg/8U1D/WPn3MVmNhTvmXwcsB74tnOu0p/1dSQzm4D3AnMEkAHciPfELig/I2b2K+AbeEebrQduxtvHHlCfkS4d7iIi0rCu3C0jIiKNULiLiAQghbuISABSuIuIBCCFu4hIAFK4i4gEIIW7iEgA+v+ugbCrAuGgIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(moving_average(np.array(score)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
